1   package org.apache.lucene.index;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one or more
5    * contributor license agreements.  See the NOTICE file distributed with
6    * this work for additional information regarding copyright ownership.
7    * The ASF licenses this file to You under the Apache License, Version 2.0
8    * (the "License"); you may not use this file except in compliance with
9    * the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  import java.io.IOException;
21  
22  import org.apache.lucene.analysis.MockAnalyzer;
23  import org.apache.lucene.document.Document;
24  import org.apache.lucene.document.Field;
25  import org.apache.lucene.document.FieldType;
26  import org.apache.lucene.document.StoredField;
27  import org.apache.lucene.document.StringField;
28  import org.apache.lucene.document.TextField;
29  import org.apache.lucene.store.Directory;
30  import org.apache.lucene.util.FailOnNonBulkMergesInfoStream;
31  import org.apache.lucene.util.LuceneTestCase;
32  import org.junit.Test;
33  
34  public class TestConsistentFieldNumbers extends LuceneTestCase {
35  
36    @Test
37    public void testSameFieldNumbersAcrossSegments() throws Exception {
38      for (int i = 0; i < 2; i++) {
39        Directory dir = newDirectory();
40        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
41                                                     .setMergePolicy(NoMergePolicy.INSTANCE));
42  
43        Document d1 = new Document();
44        d1.add(new StringField("f1", "first field", Field.Store.YES));
45        d1.add(new StringField("f2", "second field", Field.Store.YES));
46        writer.addDocument(d1);
47  
48        if (i == 1) {
49          writer.close();
50          writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
51                                           .setMergePolicy(NoMergePolicy.INSTANCE));
52        } else {
53          writer.commit();
54        }
55  
56        Document d2 = new Document();
57        FieldType customType2 = new FieldType(TextField.TYPE_STORED);
58        customType2.setStoreTermVectors(true);
59        d2.add(new TextField("f2", "second field", Field.Store.NO));
60        d2.add(new Field("f1", "first field", customType2));
61        d2.add(new TextField("f3", "third field", Field.Store.NO));
62        d2.add(new TextField("f4", "fourth field", Field.Store.NO));
63        writer.addDocument(d2);
64  
65        writer.close();
66  
67        SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
68        assertEquals(2, sis.size());
69  
70        FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
71        FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
72  
73        assertEquals("f1", fis1.fieldInfo(0).name);
74        assertEquals("f2", fis1.fieldInfo(1).name);
75        assertEquals("f1", fis2.fieldInfo(0).name);
76        assertEquals("f2", fis2.fieldInfo(1).name);
77        assertEquals("f3", fis2.fieldInfo(2).name);
78        assertEquals("f4", fis2.fieldInfo(3).name);
79  
80        writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
81        writer.forceMerge(1);
82        writer.close();
83  
84        sis = SegmentInfos.readLatestCommit(dir);
85        assertEquals(1, sis.size());
86  
87        FieldInfos fis3 = IndexWriter.readFieldInfos(sis.info(0));
88  
89        assertEquals("f1", fis3.fieldInfo(0).name);
90        assertEquals("f2", fis3.fieldInfo(1).name);
91        assertEquals("f3", fis3.fieldInfo(2).name);
92        assertEquals("f4", fis3.fieldInfo(3).name);
93  
94  
95        dir.close();
96      }
97    }
98  
99    @Test
100   public void testAddIndexes() throws Exception {
101     Directory dir1 = newDirectory();
102     Directory dir2 = newDirectory();
103     IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig(new MockAnalyzer(random()))
104                                                  .setMergePolicy(NoMergePolicy.INSTANCE));
105 
106     Document d1 = new Document();
107     d1.add(new TextField("f1", "first field", Field.Store.YES));
108     d1.add(new TextField("f2", "second field", Field.Store.YES));
109     writer.addDocument(d1);
110 
111     writer.close();
112     writer = new IndexWriter(dir2, newIndexWriterConfig(new MockAnalyzer(random()))
113                                      .setMergePolicy(NoMergePolicy.INSTANCE));
114 
115     Document d2 = new Document();
116     FieldType customType2 = new FieldType(TextField.TYPE_STORED);
117     customType2.setStoreTermVectors(true);
118     d2.add(new TextField("f2", "second field", Field.Store.YES));
119     d2.add(new Field("f1", "first field", customType2));
120     d2.add(new TextField("f3", "third field", Field.Store.YES));
121     d2.add(new TextField("f4", "fourth field", Field.Store.YES));
122     writer.addDocument(d2);
123 
124     writer.close();
125 
126     writer = new IndexWriter(dir1, newIndexWriterConfig(new MockAnalyzer(random()))
127                                      .setMergePolicy(NoMergePolicy.INSTANCE));
128     writer.addIndexes(dir2);
129     writer.close();
130 
131     SegmentInfos sis = SegmentInfos.readLatestCommit(dir1);
132     assertEquals(2, sis.size());
133 
134     FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
135     FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
136 
137     assertEquals("f1", fis1.fieldInfo(0).name);
138     assertEquals("f2", fis1.fieldInfo(1).name);
139     // make sure the ordering of the "external" segment is preserved
140     assertEquals("f2", fis2.fieldInfo(0).name);
141     assertEquals("f1", fis2.fieldInfo(1).name);
142     assertEquals("f3", fis2.fieldInfo(2).name);
143     assertEquals("f4", fis2.fieldInfo(3).name);
144 
145     dir1.close();
146     dir2.close();
147   }
148   
149   public void testFieldNumberGaps() throws IOException {
150     int numIters = atLeast(13);
151     for (int i = 0; i < numIters; i++) {
152       Directory dir = newDirectory();
153       {
154         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
155                                                     .setMergePolicy(NoMergePolicy.INSTANCE));
156         Document d = new Document();
157         d.add(new TextField("f1", "d1 first field", Field.Store.YES));
158         d.add(new TextField("f2", "d1 second field", Field.Store.YES));
159         writer.addDocument(d);
160         writer.close();
161         SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
162         assertEquals(1, sis.size());
163         FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
164         assertEquals("f1", fis1.fieldInfo(0).name);
165         assertEquals("f2", fis1.fieldInfo(1).name);
166       }
167       
168 
169       {
170         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
171                                                     .setMergePolicy(NoMergePolicy.INSTANCE));
172         Document d = new Document();
173         d.add(new TextField("f1", "d2 first field", Field.Store.YES));
174         d.add(new StoredField("f3", new byte[] { 1, 2, 3 }));
175         writer.addDocument(d);
176         writer.close();
177         SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
178         assertEquals(2, sis.size());
179         FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
180         FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
181         assertEquals("f1", fis1.fieldInfo(0).name);
182         assertEquals("f2", fis1.fieldInfo(1).name);
183         assertEquals("f1", fis2.fieldInfo(0).name);
184         assertNull(fis2.fieldInfo(1));
185         assertEquals("f3", fis2.fieldInfo(2).name);
186       }
187 
188       {
189         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
190                                                     .setMergePolicy(NoMergePolicy.INSTANCE));
191         Document d = new Document();
192         d.add(new TextField("f1", "d3 first field", Field.Store.YES));
193         d.add(new TextField("f2", "d3 second field", Field.Store.YES));
194         d.add(new StoredField("f3", new byte[] { 1, 2, 3, 4, 5 }));
195         writer.addDocument(d);
196         writer.close();
197         SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
198         assertEquals(3, sis.size());
199         FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
200         FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
201         FieldInfos fis3 = IndexWriter.readFieldInfos(sis.info(2));
202         assertEquals("f1", fis1.fieldInfo(0).name);
203         assertEquals("f2", fis1.fieldInfo(1).name);
204         assertEquals("f1", fis2.fieldInfo(0).name);
205         assertNull(fis2.fieldInfo(1));
206         assertEquals("f3", fis2.fieldInfo(2).name);
207         assertEquals("f1", fis3.fieldInfo(0).name);
208         assertEquals("f2", fis3.fieldInfo(1).name);
209         assertEquals("f3", fis3.fieldInfo(2).name);
210       }
211 
212       {
213         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
214                                                     .setMergePolicy(NoMergePolicy.INSTANCE));
215         writer.deleteDocuments(new Term("f1", "d1"));
216         // nuke the first segment entirely so that the segment with gaps is
217         // loaded first!
218         writer.forceMergeDeletes();
219         writer.close();
220       }
221 
222       IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
223                                                   .setMergePolicy(new LogByteSizeMergePolicy())
224                                                   .setInfoStream(new FailOnNonBulkMergesInfoStream()));
225       writer.forceMerge(1);
226       writer.close();
227 
228       SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
229       assertEquals(1, sis.size());
230       FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
231       assertEquals("f1", fis1.fieldInfo(0).name);
232       assertEquals("f2", fis1.fieldInfo(1).name);
233       assertEquals("f3", fis1.fieldInfo(2).name);
234       dir.close();
235     }
236   }
237 
238   @Test
239   public void testManyFields() throws Exception {
240     final int NUM_DOCS = atLeast(200);
241     final int MAX_FIELDS = atLeast(50);
242 
243     int[][] docs = new int[NUM_DOCS][4];
244     for (int i = 0; i < docs.length; i++) {
245       for (int j = 0; j < docs[i].length;j++) {
246         docs[i][j] = random().nextInt(MAX_FIELDS);
247       }
248     }
249 
250     Directory dir = newDirectory();
251     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
252 
253     for (int i = 0; i < NUM_DOCS; i++) {
254       Document d = new Document();
255       for (int j = 0; j < docs[i].length; j++) {
256         d.add(getField(docs[i][j]));
257       }
258 
259       writer.addDocument(d);
260     }
261 
262     writer.forceMerge(1);
263     writer.close();
264 
265     SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
266     for (SegmentCommitInfo si : sis) {
267       FieldInfos fis = IndexWriter.readFieldInfos(si);
268 
269       for (FieldInfo fi : fis) {
270         Field expected = getField(Integer.parseInt(fi.name));
271         assertEquals(expected.fieldType().indexOptions(), fi.getIndexOptions());
272         assertEquals(expected.fieldType().storeTermVectors(), fi.hasVectors());
273       }
274     }
275 
276     dir.close();
277   }
278 
279   private Field getField(int number) {
280     int mode = number % 16;
281     String fieldName = "" + number;
282     FieldType customType = new FieldType(TextField.TYPE_STORED);
283     
284     FieldType customType2 = new FieldType(TextField.TYPE_STORED);
285     customType2.setTokenized(false);
286     
287     FieldType customType3 = new FieldType(TextField.TYPE_NOT_STORED);
288     customType3.setTokenized(false);
289     
290     FieldType customType4 = new FieldType(TextField.TYPE_NOT_STORED);
291     customType4.setTokenized(false);
292     customType4.setStoreTermVectors(true);
293     customType4.setStoreTermVectorOffsets(true);
294     
295     FieldType customType5 = new FieldType(TextField.TYPE_NOT_STORED);
296     customType5.setStoreTermVectors(true);
297     customType5.setStoreTermVectorOffsets(true);
298 
299     FieldType customType6 = new FieldType(TextField.TYPE_STORED);
300     customType6.setTokenized(false);
301     customType6.setStoreTermVectors(true);
302     customType6.setStoreTermVectorOffsets(true);
303 
304     FieldType customType7 = new FieldType(TextField.TYPE_NOT_STORED);
305     customType7.setTokenized(false);
306     customType7.setStoreTermVectors(true);
307     customType7.setStoreTermVectorOffsets(true);
308 
309     FieldType customType8 = new FieldType(TextField.TYPE_STORED);
310     customType8.setTokenized(false);
311     customType8.setStoreTermVectors(true);
312     customType8.setStoreTermVectorPositions(true);
313 
314     FieldType customType9 = new FieldType(TextField.TYPE_NOT_STORED);
315     customType9.setStoreTermVectors(true);
316     customType9.setStoreTermVectorPositions(true);
317 
318     FieldType customType10 = new FieldType(TextField.TYPE_STORED);
319     customType10.setTokenized(false);
320     customType10.setStoreTermVectors(true);
321     customType10.setStoreTermVectorPositions(true);
322 
323     FieldType customType11 = new FieldType(TextField.TYPE_NOT_STORED);
324     customType11.setTokenized(false);
325     customType11.setStoreTermVectors(true);
326     customType11.setStoreTermVectorPositions(true);
327 
328     FieldType customType12 = new FieldType(TextField.TYPE_STORED);
329     customType12.setStoreTermVectors(true);
330     customType12.setStoreTermVectorOffsets(true);
331     customType12.setStoreTermVectorPositions(true);
332 
333     FieldType customType13 = new FieldType(TextField.TYPE_NOT_STORED);
334     customType13.setStoreTermVectors(true);
335     customType13.setStoreTermVectorOffsets(true);
336     customType13.setStoreTermVectorPositions(true);
337 
338     FieldType customType14 = new FieldType(TextField.TYPE_STORED);
339     customType14.setTokenized(false);
340     customType14.setStoreTermVectors(true);
341     customType14.setStoreTermVectorOffsets(true);
342     customType14.setStoreTermVectorPositions(true);
343 
344     FieldType customType15 = new FieldType(TextField.TYPE_NOT_STORED);
345     customType15.setTokenized(false);
346     customType15.setStoreTermVectors(true);
347     customType15.setStoreTermVectorOffsets(true);
348     customType15.setStoreTermVectorPositions(true);
349     
350     switch (mode) {
351       case 0: return new Field(fieldName, "some text", customType);
352       case 1: return new TextField(fieldName, "some text", Field.Store.NO);
353       case 2: return new Field(fieldName, "some text", customType2);
354       case 3: return new Field(fieldName, "some text", customType3);
355       case 4: return new Field(fieldName, "some text", customType4);
356       case 5: return new Field(fieldName, "some text", customType5);
357       case 6: return new Field(fieldName, "some text", customType6);
358       case 7: return new Field(fieldName, "some text", customType7);
359       case 8: return new Field(fieldName, "some text", customType8);
360       case 9: return new Field(fieldName, "some text", customType9);
361       case 10: return new Field(fieldName, "some text", customType10);
362       case 11: return new Field(fieldName, "some text", customType11);
363       case 12: return new Field(fieldName, "some text", customType12);
364       case 13: return new Field(fieldName, "some text", customType13);
365       case 14: return new Field(fieldName, "some text", customType14);
366       case 15: return new Field(fieldName, "some text", customType15);
367       default: return null;
368     }
369   }
370 }